package fetcher

import (
	"bufio"
	"fmt"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding"
	"golang.org/x/text/encoding/unicode"
	"golang.org/x/text/transform"
	"io/ioutil"
	"net/http"
	"time"
)

// 100毫秒执行一次请求
var rateLimiter = time.Tick(100 * time.Millisecond)

func Fetcher(url string)([]byte,error){
	<- rateLimiter
	req,_ := http.NewRequest(http.MethodGet,url,nil)
	var httpClient = http.Client{}
	rsp,err := httpClient.Do(req)
	if err != nil {
		return nil,err
	}
	defer rsp.Body.Close()
	if rsp.StatusCode != 200 {
		return  nil,fmt.Errorf("请求出差！返回码【%d】",rsp.StatusCode)
	}
	bufBody := bufio.NewReader(rsp.Body)
	utf8Reader := transform.NewReader(bufBody,determineEncoding(bufBody).NewDecoder())
	body,err := ioutil.ReadAll(utf8Reader)
	return body,err
}


func determineEncoding(r *bufio.Reader) encoding.Encoding {
	data,err := r.Peek(1024)
	if err != nil {
		return  unicode.UTF8
	}
	e,_,_ := charset.DetermineEncoding(data,"")
	return e
}
/*
func FindUrl(){
	data := RequstData()
	re := regexp.MustCompile(`<a href="(/info-detail-\w+.html)" title="[\s\S]*?"[^>]*>([^<]+)</a>`)
	txt := string(data)
	fmt.Printf("txt length is %d\n",len(txt))

	dir := make(map[string]bool,0)
	matcheds := re.FindAllSubmatch(data,-1)
	for _,item := range  matcheds {
		title :=  strings.TrimSpace(string(item[2]))
		if dir[title] {
			continue
		}
		dir[title] = true
		fmt.Printf("URL: %s , Title:%s\n",string(item[1]),title)
	}
}
 */
